

#include"DictProducer.h"
#include"SloSplitTool.h"
using std::ifstream;
using std::cout;
using std::cerr;
using std::stringstream;
using std::istringstream;
using std::ofstream;
DictProducer::DictProducer(string path)
{
    DIR * pdir=opendir(path.c_str());
     if (pdir == NULL) {
        perror("opendir");
        return;
    }
    struct dirent* pdirent;
    while((pdirent=readdir(pdir))){
        if(string(pdirent->d_name)!="."&&string(pdirent->d_name)!=".."){
        _files.push_back(string(path).append("/").append(pdirent->d_name));
        }
    }
    closedir(pdir);
}
void DictProducer::buildEnDict(){
    //打开文件
    ifstream ifs("/home/zzh/cpp/project/v1_project/yuliao/english.txt");
    if(ifs.is_open()){
        cout<<"file open success!\n";
    }else{
        perror("file open fail\n");
        return;
    }
    //读入整个文本文件
    stringstream buf;
    buf<<ifs.rdbuf();
    //清洗
    string content=buf.str();
    cleanEnContent(content);
    //分词以及词频统计
    map<string,int> dictTotal;
    istringstream ibuf(content);
    string word;
    while(ibuf>>word){
        ++dictTotal[word];
    }
    //导入_dict方便建立索引
    for(auto ele:dictTotal){
        _dict.push_back(ele);
    }
    ofstream ofs("/home/zzh/cpp/project/v1_project/search/data/dict.dat",std::ios_base::app);
    for(auto ele:_dict){
        string line=ele.first+" "+std::to_string(ele.second)+"\n";
        ofs.write(line.c_str(),line.size());
    }
}
void DictProducer::buildCnDIct(){

    map<string,int>wordTotal;
    for(auto ele:_files){
        //打开文件

        ifstream ifs(ele);
        if(ifs.is_open()){
            cout<<"file open success!\n";
        }else{
            perror("file open fail\n");
            return;
        }
        //读入整个文本文件
        stringstream buf;
        buf<<ifs.rdbuf();
        //清洗
        string content=buf.str();
        cleanCnContent(content);
        //分词
        vector<string> wordlist=SloSplitTool::getInstance()->cut(content);
        //词频统计
        for(auto ele:wordlist){
            ++wordTotal[ele];
        }
    }
    //导入_dict
    for(auto ele:wordTotal){
        _dict.push_back(ele);
    }
    ofstream ofs("/home/zzh/cpp/project/v1_project/search/data/dict.dat",std::ios_base::app);
    for(auto ele:_dict){
        string line=ele.first+" "+std::to_string(ele.second)+"\n";
        ofs.write(line.c_str(),line.size());
    }
}
void DictProducer::createIndex(){
      //建立索引
    for(size_t i=0;i<_dict.size();++i){
        string word=_dict[i].first;
        for(size_t idx=0;idx<word.size();){
            if((word[idx]&0x80)==0){
                _index[word.substr(idx,1)].insert(i);
                ++idx;
            }else{
                size_t off=nByetesCode(word[idx]);
                _index[word.substr(idx,off)].insert(i);
                idx+=off;
            }
        }
    }
    ofstream ofs("/home/zzh/cpp/project/v1_project/search/data/dictIndex.dat",std::ios_base::app);
    for(auto ele:_index){
        string line=ele.first+" ";
        for(auto eles:ele.second){
            line+=(" "+std::to_string(eles));
        }
        line+=+"\n";
        ofs.write(line.c_str(),line.size());
    }

}
void DictProducer::cleanEnContent(string &content){
    for(auto &ele:content){
        if(isupper(ele)){
            ele=tolower(ele);
        }
        if(ispunct(ele)){
            ele=' ';
        }
    }
}
void DictProducer::cleanCnContent(string &content){
    auto it=content.begin();
    for(;it!=content.end();){
        if(*it=='\n'||*it=='\r'||*it==' '){
            it=content.erase(it);
        }else{
            ++it;
        }
    }
}

size_t DictProducer::nByetesCode(const char ch){
    if(ch&(1<<7)){
        size_t nByetes=1;
        for(int idx=6;idx>0;--idx){
            if(ch&(1<<idx)){
                ++nByetes;
            }else{
                break;
            }
        }
        return nByetes;
    }
    return 1;
}

void DictProducer::printDict(){
    for(auto ele:_dict){
        cout<<ele.first<<" : "<<ele.second<<"\n";
    }
}